/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.ipc;

import org.apache.commons.logging.*;

import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.net.NetUtils;

import java.util.Random;
import java.io.DataInput;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.SocketTimeoutException;

import junit.framework.TestCase;

import org.apache.hadoop.conf.Configuration;
import org.junit.Ignore;
import org.junit.Test;
import static org.junit.Assert.*;



/** Unit tests for IPC. */
public class TestIPC {
  public static final Log LOG =
    LogFactory.getLog(TestIPC.class);
  
  final private static Configuration conf = new Configuration();
  final static private int PING_INTERVAL = 1000;
  final static private int MIN_SLEEP_TIME = 1000;
  
  static {
    Client.setPingInterval(conf, PING_INTERVAL);
    conf.setInt(Server.IPC_SERVER_RPC_READ_THREADS_KEY, 3);
    conf.setInt(Server.IPC_SERVER_CLIENT_IDLETHRESHOLD, 0);
    conf.setInt(Server.IPC_SERVER_CLIENT_CONN_MAXIDLETIME, 150);
  }

  private static final Random RANDOM = new Random();

  private static final String ADDRESS = "0.0.0.0";

  private static class TestServer extends Server {
    private boolean sleep;

    public TestServer(int handlerCount, boolean sleep) throws IOException {
      super(ADDRESS, 0, LongWritable.class, handlerCount, conf);
      this.sleep = sleep;
    }

    @Override
    public Writable call(Class<?> protocol, Writable param, long receiveTime)
        throws IOException {
      if (sleep) {
        // sleep a bit
        try {
          Thread.sleep(RANDOM.nextInt(PING_INTERVAL) + MIN_SLEEP_TIME);
        } catch (InterruptedException e) {
        }
      }
      return param; // echo param as result
    }
  }

  private static class SerialCaller extends Thread {
    private Client client;
    private InetSocketAddress server;
    private int count;
    private int repeatCount;
    private boolean failed;
    // We use socketTime as a hacky way to get separate sockets to the same address
    private int lowSocketTimeout;
    private int highSocketTimeout;

    public SerialCaller(Client client, InetSocketAddress server, int count, int repeatCount,
        int lowSocketTimeout, int highSocketTimeout) {
      this.client = client;
      this.server = server;
      this.count = count;
      this.repeatCount = repeatCount;
      this.lowSocketTimeout = lowSocketTimeout;
      this.highSocketTimeout = highSocketTimeout;
    }

    public void run() {
      for (int i = 0; i < count; i++) {
        int socketTimeout;
        if (lowSocketTimeout == highSocketTimeout) {
          socketTimeout = lowSocketTimeout;
        } else {
          socketTimeout = lowSocketTimeout
              + RANDOM.nextInt(highSocketTimeout - lowSocketTimeout);
        }
        for (int j = 0; j < repeatCount; j++) {
          try {
            LongWritable param = new LongWritable(RANDOM.nextLong());
            LongWritable value = (LongWritable) client.call(param, server,
                null, null, socketTimeout, false);
            if (!param.equals(value)) {
              LOG.fatal("Call failed!");
              failed = true;
              break;
            }
          } catch (Exception e) {
            LOG.fatal("Caught: " + StringUtils.stringifyException(e));
            failed = true;
          }
        }
      }
    }
  }

  private static class ParallelCaller extends Thread {
    private Client client;
    private int count;
    private InetSocketAddress[] addresses;
    private boolean failed;
    
    public ParallelCaller(Client client, InetSocketAddress[] addresses,
                          int count) {
      this.client = client;
      this.addresses = addresses;
      this.count = count;
    }

    public void run() {
      for (int i = 0; i < count; i++) {
        try {
          Writable[] params = new Writable[addresses.length];
          for (int j = 0; j < addresses.length; j++)
            params[j] = new LongWritable(RANDOM.nextLong());
          Writable[] values = client.call(params, addresses, null, null);
          for (int j = 0; j < addresses.length; j++) {
            if (!params[j].equals(values[j])) {
              LOG.fatal("Call failed!");
              failed = true;
              break;
            }
          }
        } catch (Exception e) {
          LOG.fatal("Caught: " + StringUtils.stringifyException(e));
          failed = true;
        }
      }
    }
  }

  @Test
  public void testSerial() throws Exception {
    testSerial(3, false, 2, 5, 100, 1, 0, 0);
    testSerial(3, true, 2, 5, 10, 1, 0, 0);
  }

  /**
   * This is a simple benchmark which makes some traffic to both of IPC Server's
   * listener and reader threads' so we can measure performance improvement of
   * IPC's socket accept/read mechanism.
   * 
   * We create 32 clients all connecting to the same IPC server with 3 request
   * reader threads. Every client sends 1024 requests using one connection and
   * then moves on to a next one. In that way, both accept thread and reader
   * threads get consistent loads.
   * 
   * The approach we use for a clients to move on to separate connections is by
   * changing the socket timeout. Our IPC client will create a different
   * connection if the socket timeout setting is different.
   * 
   * @throws Exception
   */
  @Ignore
  @Test
  public void testSerialBenchmark1() throws Exception {
    conf.setInt("ipc.client.connection.maxidletime", 1);

    long startTime = System.currentTimeMillis();
    testSerial(32, false, 8, 32, 32, 1024, 60000, 60000 + 8);
    long endTime = System.currentTimeMillis();
    System.out.println("============== Benchmark Took: "
        + (endTime - startTime) + " ms ===================");

    conf.setInt("ipc.client.connection.maxidletime", 10000);

  }

  public void testSerial(int handlerCount, boolean handlerSleep,
      int clientCount, int callerCount, int callCount, int repeatCount,
      int lowSocketTimeout, int highSocketTimeout) throws Exception {
    Server server = new TestServer(handlerCount, handlerSleep);
    InetSocketAddress addr = NetUtils.getConnectAddress(server);
    server.start();

    Client[] clients = new Client[clientCount];
    for (int i = 0; i < clientCount; i++) {
      clients[i] = new Client(LongWritable.class, conf);
    }
    
    SerialCaller[] callers = new SerialCaller[callerCount];
    for (int i = 0; i < callerCount; i++) {
      callers[i] = new SerialCaller(clients[i % clientCount], addr, callCount,
          repeatCount, lowSocketTimeout, highSocketTimeout);
      callers[i].start();
    }
    for (int i = 0; i < callerCount; i++) {
      callers[i].join();
      assertFalse(callers[i].failed);
    }
    for (int i = 0; i < clientCount; i++) {
      clients[i].stop();
    }
    server.stop();
  }
  
  private void waitNMilliSecond(long startTime, long timeout) {
    long timeToSleep = timeout + startTime - System
        .currentTimeMillis();
    if (timeToSleep <= 0) {
      return;
    }
    DFSTestUtil.waitNMilliSecond((int) timeToSleep);
  }

  @Test
  public void testCleaningIdleConnections() throws Exception {
    conf.setInt(Server.IPC_SERVER_RPC_READ_THREADS_KEY, 1);

    Server server = new TestServer(2, false);
    server.cleanupInterval = 10;
    server.PURGE_INTERVAL = 100;
    InetSocketAddress addr = NetUtils.getConnectAddress(server);
    server.start();
    
    Configuration clientConf = new Configuration(conf);
    // Make sure the server timeout triggers first.
    clientConf.setInt(Server.IPC_SERVER_CLIENT_CONN_MAXIDLETIME, 10000);
    
    Client client1 = new Client(LongWritable.class, clientConf);
    Client client2 = new Client(LongWritable.class, clientConf);

    LongWritable param = new LongWritable(0);
    client1.call(param, addr, null, null, 100000, false);
    DFSTestUtil.waitNMilliSecond(150);
    assertEquals(1, server.getNumOpenConnections());
    client2.call(param, addr, null, null, 100000, false);
    assertEquals(2, server.getNumOpenConnections());
    DFSTestUtil.waitNMilliSecond(220);
    assertEquals(1, server.getNumOpenConnections());
    DFSTestUtil.waitNMilliSecond(150);
    assertEquals(0, server.getNumOpenConnections());
    
    assertTrue(server.connectionSet.ifConnectionsClean());
    
    // Make sure remove happens in the same buckets by having many clients so
    // there will sure be clients in the same buckets.
    Client[] clients1 = new Client[5];
    Client[] clients2 = new Client[5];
    Client[] clients3 = new Client[5];
    
    for (int i = 0; i < 5; i++) {
      clients1[i] = new Client(LongWritable.class, clientConf);
      clients2[i] = new Client(LongWritable.class, clientConf);
      clients3[i] = new Client(LongWritable.class, clientConf);
    }

    long startTime = System.currentTimeMillis();
    for (int i = 0; i < 5; i++) {
      clients1[i].call(param, addr, null, null, 100000, false);
    }
    assertEquals(5, server.getNumOpenConnections());
    waitNMilliSecond(startTime, 90);
    startTime = System.currentTimeMillis();
    for (int i = 0; i < 5; i++) {
      clients2[i].call(param, addr, null, null, 100000, false);
    }
    assertEquals(10, server.getNumOpenConnections());
    waitNMilliSecond(startTime, 90);
    startTime = System.currentTimeMillis();
    for (int i = 0; i < 5; i++) {
      clients3[i].call(param, addr, null, null, 100000, false);
    }
    assertEquals(15, server.getNumOpenConnections());
    waitNMilliSecond(startTime, 90);
    startTime = System.currentTimeMillis();
    // Renew first batch of calls so that they don't expire
    for (int i = 0; i < 5; i++) {
      clients1[i].call(param, addr, null, null, 100000, false);
    }
    assertEquals(15, server.getNumOpenConnections());

    waitNMilliSecond(startTime, 90);
    startTime = System.currentTimeMillis();
    assertEquals(15, server.getNumOpenConnections());
    // renew the second batch of calls
    for (int i = 0; i < 5; i++) {
      clients2[i].call(param, addr, null, null, 100000, false);
    }
    // Wait for clients are cleared as expected.
    DFSTestUtil.waitNMilliSecond(180);
    assertEquals(10, server.getNumOpenConnections());
    DFSTestUtil.waitNMilliSecond(200);
    assertEquals(0, server.getNumOpenConnections());
    
    assertTrue(server.connectionSet.ifConnectionsClean());
    
    server.stop();
  }
  
  @Test
  public void testParallel() throws Exception {
    testParallel(10, false, 2, 4, 2, 4, 100);
  }

  public void testParallel(int handlerCount, boolean handlerSleep,
                           int serverCount, int addressCount,
                           int clientCount, int callerCount, int callCount)
    throws Exception {
    Server[] servers = new Server[serverCount];
    for (int i = 0; i < serverCount; i++) {
      servers[i] = new TestServer(handlerCount, handlerSleep);
      servers[i].start();
    }

    InetSocketAddress[] addresses = new InetSocketAddress[addressCount];
    for (int i = 0; i < addressCount; i++) {
      addresses[i] = NetUtils.getConnectAddress(servers[i%serverCount]);
    }

    Client[] clients = new Client[clientCount];
    for (int i = 0; i < clientCount; i++) {
      clients[i] = new Client(LongWritable.class, conf);
    }
    
    ParallelCaller[] callers = new ParallelCaller[callerCount];
    for (int i = 0; i < callerCount; i++) {
      callers[i] =
        new ParallelCaller(clients[i%clientCount], addresses, callCount);
      callers[i].start();
    }
    for (int i = 0; i < callerCount; i++) {
      callers[i].join();
      assertFalse(callers[i].failed);
    }
    for (int i = 0; i < clientCount; i++) {
      clients[i].stop();
    }
    for (int i = 0; i < serverCount; i++) {
      servers[i].stop();
    }
  }
	
  @Test
  public void testStandAloneClient() throws Exception {
    testParallel(10, false, 2, 4, 2, 4, 100);
    Client client = new Client(LongWritable.class, conf);
    InetSocketAddress address = new InetSocketAddress("127.0.0.1", 10);
    try {
      client.call(new LongWritable(RANDOM.nextLong()),
              address, null, null, 0, false);
      fail("Expected an exception to have been thrown");
    } catch (IOException e) {
      String message = e.getMessage();
      String addressText = address.toString();
      assertTrue("Did not find "+addressText+" in "+message,
              message.contains(addressText));
      Throwable cause=e.getCause();
      assertNotNull("No nested exception in "+e,cause);
      String causeText=cause.getMessage();
      assertTrue("Did not find " + causeText + " in " + message,
              message.contains(causeText));
    }
  }

  private static class LongErrorWritable extends LongWritable {
    private final static String ERR_MSG =
      "Come across an exception while reading";

    LongErrorWritable() {}

    LongErrorWritable(long longValue) {
      super(longValue);
    }

    public void readFields(DataInput in) throws IOException {
      super.readFields(in);
      throw new IOException(ERR_MSG);
    }
  }

  @Test
  public void testErrorClient() throws Exception {
    // start server
    Server server = new TestServer(1, false);
    InetSocketAddress addr = NetUtils.getConnectAddress(server);
    server.start();

    // start client
    Client client = new Client(LongErrorWritable.class, conf);
    try {
      client.call(new LongErrorWritable(RANDOM.nextLong()),
          addr, null, null, 0, false);
      fail("Expected an exception to have been thrown");
    } catch (IOException e) {
      // check error
      Throwable cause = e.getCause();
      assertTrue(cause instanceof IOException);
      assertEquals(LongErrorWritable.ERR_MSG, cause.getMessage());
    }
  }

  @Test
  public void testIpcTimeout() throws Exception {
    // start server
    Server server = new TestServer(1, true);
    InetSocketAddress addr = NetUtils.getConnectAddress(server);
    server.start();

    // start client
    Client client = new Client(LongWritable.class, conf);
    // set timeout to be less than MIN_SLEEP_TIME
    try {
      client.call(new LongWritable(RANDOM.nextLong()),
              addr, null, null, MIN_SLEEP_TIME/2, false);
      fail("Expected an exception to have been thrown");
    } catch (SocketTimeoutException e) {
      LOG.info("Get a SocketTimeoutException ", e);
    }
    // set timeout to be bigger than 3*ping interval
    client.call(new LongWritable(RANDOM.nextLong()),
        addr, null, null, 3*PING_INTERVAL+MIN_SLEEP_TIME, false);
  }

	public static void main(String[] args) throws Exception {

    //new TestIPC("test").testSerial(5, false, 2, 10, 1000);

	  if (args.length >= 1 && args[0].equals("benchmark")) {
	    System.out.println("args: " + args[0]);
	    new TestIPC().testSerialBenchmark1();
	  } else {
	    new TestIPC().testParallel(10, false, 2, 4, 2, 4, 1000);
	  }
  }

}
